import time
import logging, json
import os
import pymysql
from scrapy.utils.project import get_project_settings
json_path = './followingJsons'
user_name_dirs = os.listdir(json_path)
all_user = {}

SETTINGS = get_project_settings()
mysql_conn = pymysql.connect(host=SETTINGS['MYSQL_HOST'],
                       port=SETTINGS['MYSQL_PORT'],
                       user=SETTINGS['MYSQL_USER'],
                       password=SETTINGS['MYSQL_PASS'],
                       db=SETTINGS['MYSQL_DB'])

cursor = mysql_conn.cursor()
logger = logging.getLogger(__name__)


def save_to_db(user):
    sql = "INSERT INTO {} (user_id, username, follower, description) " \
          "VALUES ( {}, '{}', '{}', '{}')" \
        .format('following_user', user['id'], user['name'], user['follower'], user['description'])
    try:
        cursor.execute(sql)
        mysql_conn.commit()
        print("Add User: {{ name: {}, user_id: {} }}".format(user['name'], user['id']))
    except Exception as e:
        if 'Duplicate entry' in str(e):
            print("User {{ name: {}, user_id: {} }} is already exist.".format(user['name'], user['id']))
        else:
            print(str(e))
        mysql_conn.rollback()

# if __name__ == '__main__':


for user_dir in user_name_dirs:
    follower = user_dir
    temp_path = '{}/{}'.format(json_path, user_dir)
    file_names = os.listdir(temp_path)
    for fn in file_names:
        if not fn.endswith('.txt'):
            continue
        real_path = '{}/{}'.format(temp_path, fn)
        with open(real_path, 'r', encoding="utf-8") as f:
            data = json.load(f)
        user_items = data['data']['user']['following_timeline']['timeline']['instructions'][-1]['entries']
        user_list = []
        for item in user_items[:-2]:
            user = {}
            user_id_str = item['entryId']
            if 'user-' not in user_id_str:
                continue
            user['id'] = user_id_str.replace('user-', '')
            user['name'] = item['content']['itemContent']['user']['legacy']['name']
            user['description'] = item['content']['itemContent']['user']['legacy']['description']
            user['follower'] = follower
            save_to_db(user)
            user_list.append(user)
    all_user[follower] = user_list
print(all_user)
